#coding:utf-8
import json
import math
import os
import sys
import time
from pyspark.sql import SparkSession
from pyspark.sql.functions import *
from obs import *

def getObsClient(nnList):
    AK = nnList[0]
    SK = nnList[1]
    server = nnList[2]

    # 创建ObsClient实例
    obsClient = ObsClient(
        access_key_id=AK,    
        secret_access_key=SK,    
        server=server
    )
    return obsClient

def dropStartWithSlash(content):
    if str(content).startswith('/'):
        content=content[1:]
    return content

def moveFiles(obsClient, bucketName,flumeDataPath, hiveTempPath, currProtocol, limitNum):
    desc_count=getFileNum(obsClient,bucketName,hiveTempPath)    
    if desc_count > limitNum:
        return
    errorLog=[]
    mkObjDir(obsClient, bucketName, hiveTempPath + '/')
    resp = obsClient.listObjects(bucketName, flumeDataPath + '/', delimiter='/')
    for prefix in resp.body.commonPrefixs:
        dateresp = obsClient.listObjects(bucketName, str(prefix.prefix) + currProtocol)
        for obj in dateresp.body.contents:
            if obj.size == 0:
                continue
            if desc_count > limitNum:
                break
            oriKey=obj.key
            if not oriKey.endswith('.snappy'):
                continue
            fileMetaArray=oriKey.split('/')
            fileName=fileMetaArray[len(fileMetaArray) - 1]
            descKey=hiveTempPath +'/'+ fileName
            print("copy before key is " + oriKey + ", copy desc key is " + descKey)
            copyresp = obsClient.copyObject(bucketName, oriKey, bucketName, descKey)
            if copyresp.status < 300:
                desc_count=desc_count+1
                print("delete file is " + oriKey)
                deleteresp = obsClient.deleteObject(bucketName, oriKey)
                if deleteresp.status >= 300:
                    errorLog.append('errorCode:'+ deleteresp.errorCode)
                    errorLog.append('errorMessage' + deleteresp.errorMessage)
                    obsClient.deleteObject(bucketName, oriKey)
            else:
                errorLog.append('errorCode:' + copyresp.errorCode)
                errorLog.append('errorMessage:' + copyresp.errorMessage)
        if len(errorLog) > 0:
            print(errorLog)

def getFileNum(obsClient,bucketName,path):
    desc_count=0
    resp = obsClient.listObjects(bucketName, path)
    for obj in resp.body.contents:
        if obj.size > 0:
            desc_count=desc_count+1
    return desc_count

def getRepartitionByFileNum(client, bucketName, hiveTempPath, stdValue):
    filecount = getFileNum(client, bucketName, hiveTempPath)
    if(stdValue < 1): stdValue = 1
    print("Total file number of folders: " + str(filecount))
    repartition = int(math.ceil(filecount / stdValue)) + 1
    if repartition < 1:
        repartition = 1
    return repartition


def backupData(obsClient, bucketName, hiveTempPath, backupDataPath, needProtocol):
    time_now = time.strftime("%Y-%m-%d")
    backupDataPath = backupDataPath + '/'  + needProtocol + '/' + time_now + '/'
    mkObjDir(obsClient, bucketName, backupDataPath) 
    errorLog = []
    resp = obsClient.listObjects(bucketName, hiveTempPath + '/')
    for obj in resp.body.contents:
        if obj.size == 0:
            continue
        oriKey=obj.key
        fileMetaArray=oriKey.split('/')
        fileName=fileMetaArray[len(fileMetaArray) - 1]
        descKey= backupDataPath + fileName 
        print("copy before key is " + oriKey + ", copy desc key is " + descKey)
        copyresp = obsClient.copyObject(bucketName, oriKey, bucketName, descKey)
        if copyresp.status < 300:
            print("delete key is " + oriKey)
            deleteresp = obsClient.deleteObject(bucketName, oriKey)
            if deleteresp.status >= 300:
                errorLog.append('errorCode:'+ deleteresp.errorCode)
                errorLog.append('errorMessage' + deleteresp.errorMessage) 
                obsClient.deleteObject(bucketName, oriKey)
        else:
            errorLog.append('errorCode:' + copyresp.errorCode)
            errorLog.append('errorMessage:' + copyresp.errorMessage)
    if len(errorLog) > 0:
        print(errorLog)


def objectExists(obsClient,bucketName,path):
    existResp=obsClient.getObject(bucketName, path)
    if existResp.errorCode == "NoSuchKey":
        return False
    else:
        return True

def mkObjDir(obsClient,bucketName,path):
    exists=objectExists(obsClient,bucketName,path)
    if not exists:
       resp = obsClient.putContent(bucketName, path, content=None)
def readFile(filename):
	f = open(filename, "r")
	data = f.read()
	f.close()
	return data
